import scrapy

from sp.items import SpItem
from scrapy.contrib.spiders import Rule,CrawlSpider
from scrapy.contrib.linkextractors import LinkExtractor

class SpSpider(CrawlSpider):
    name = 'sp'
    allowed_domains = ('www.abcd.com', 'abcd.com')
    
    start_urls = [
        "http://www.abcd.com/interest/"
    ]
    
    rules = (
        Rule(LinkExtractor(allow=('/interest/[0-9]+\.html', )), callback='parse_post'),
        Rule(LinkExtractor(allow=('/interest/\?page=[0-9]+?', )), follow=True),
    )
    
    def parse_post(self, response):
        item = SpItem()
        link  = response.url
        link_segments = link.split('/')
        postid = link_segments[len(link_segments)-1].split('.')[0]
        images = response.xpath('//img/@src').re('.*(?<!xqlogo\.jpg)$')
        title  = response.xpath('//h1[@class="title"]/text()').extract()
        
        item['link']  = link
        item['title']  = title[0]
        item['postid'] = postid
        item['image_urls'] = images
        
        return item